#include "include/indexer.h"

Indexer::Indexer() :
    word_seq_(0), doc_seq_(0), pre_word_seq_(0),
    used_time_(0), total_doc_len_(0) {
  cidx_path_ = Config::get_instance().get("cidx_path");
  cidx_file_ = Config::get_instance().get("cidx_file");
  tidx_path_ = Config::get_instance().get("tidx_path");
  tidx_file_ = Config::get_instance().get("tidx_file");

  try {
    index_compr_ = new IndexCompressor(BYTECODEEX);
    seg_ = new Segment();
    words_ht_ = new HashTable(1024 * 1024);
    doc_info_ = new DocInfo[MAX_DOC_NUM];
  } catch (const std::bad_alloc &ex) {
    cerr << "oom: " << ex.what() << endl;
    exit(-1);
  }

  for (int i = 0; i < MAX_WORDSNUM_SIZE; ++i) {
    pre_doc_seq_c_[i] = 0;
    pre_doc_seq_t_[i] = 0;

    c_index_buffer_[i].pos = 0;
    c_index_buffer_[i].size = 0;
    c_index_buffer_[i].pbuffer = NULL;

    t_index_buffer_[i].pos = 0;
    t_index_buffer_[i].size = 0;
    t_index_buffer_[i].pbuffer = NULL;
  }

  //fout.open("words.txt",ofstream::out);//test
}

Indexer::~Indexer() {
  delete index_compr_;
  delete words_ht_;
  delete []doc_info_;
  delete seg_;
  for (int i = 0; i < MAX_WORDSNUM_SIZE; ++i) {
    delete []c_index_buffer_[i].pbuffer;
    delete []t_index_buffer_[i].pbuffer;
  }
  
  //Segment::get_instance().release();
  Config::get_instance().release();
  //fout.close(); //test
}

void Indexer::index_all(const string& dir_name) {
  DIR *dp = opendir(dir_name.c_str());
  if (dp == NULL) {
    cerr << "open dir [" << dir_name << "] error" << endl;
    exit(-1);
  }
  struct dirent *de = NULL;
  char file_name[MAX_FILENAME_LEN];
  Reader* r = new Reader();

  struct timeb begin, finish;
  ftime(&begin);
  //-- start
  while (de = readdir(dp)) {
    if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
      continue;
    memset(file_name, 0, MAX_FILENAME_LEN);
    sprintf(file_name, "%s/%s", dir_name.c_str(), de->d_name);

    r->load_file(file_name);
    DocItem di;
    while (r->read_next_item(di) > 0) {
      doc_info_[doc_seq_].doc_len = di.content.length();
      doc_info_[doc_seq_].url = di.url;
      doc_info_[doc_seq_].title = di.title;
      total_doc_len_ += doc_info_[doc_seq_].doc_len;

      index_one(di.content, doc_seq_, T_CONTENT);

      index_one(di.title, doc_seq_, T_TITLE);
      ++doc_seq_;
    }
  }

  delete r;
  
  flush(c_index_buffer_, cidx_path_);
  flush(t_index_buffer_, tidx_path_);

  merge(cidx_file_, cidx_path_);
  merge(tidx_file_, tidx_path_);

  complete(); //save doc_info & words_info
  
  //-- end
  ftime(&finish);
  used_time_ = (finish.time - begin.time) * 1000 + finish.millitm
      - begin.millitm;
  //words_ht_->show();
}

void Indexer::index_one(const string& text, int& doc_seq, FieldType field) {
  if (text.length() == 0)
    return;

  vector<SegRes> res;
  vector<SegRes>::iterator it;
  int useless = 0;

  if (field == T_CONTENT) {

    //Segment::get_instance().segment(text.c_str(), res);
    seg_->segment(text.c_str(), res);
    for (it = res.begin(); it != res.end(); ++it) {
      if (!(it->type == 'C' || it->type == 'c' || it->type == 'x'
          || it->type == 'S'))
        continue;

      char* tmp = new char[it->word.length() + 1];
      strcpy(tmp, it->word.c_str());

      pre_word_seq_ = word_seq_;

      int word_id = words_ht_->insert(tmp, word_seq_);

      if (word_seq_ > MAX_WORDSNUM_SIZE) {
        cerr << "word_seq_ > MAX_WORDSNUM_SIZE" << endl;
        exit(-1);
      }

      if (word_seq_ > pre_word_seq_) {
        pre_word_seq_ = word_seq_;
      } else { //already exist
        delete []tmp;
      }
      int inter = doc_seq - pre_doc_seq_c_[word_id];
      pre_doc_seq_c_[word_id] = doc_seq;

      char tmpbuf[16]; //TODO: size
      int tmplen = 0;
      index_compr_->encode(inter, tmpbuf, tmplen, useless); // default: BYTECODEEX
      index_compr_->encode(it->times, tmpbuf, tmplen, useless);
      if (tmplen + c_index_buffer_[word_id].pos > GBUFFERLEN) {
        write_buffer(c_index_buffer_, word_id, tmpbuf, tmplen, cidx_path_);
      } else {
        if (tmplen + c_index_buffer_[word_id].pos
            > c_index_buffer_[word_id].size) { //not enough space
          char* pstr = NULL;
          try {
            pstr = new char[c_index_buffer_[word_id].size + LBUFFERLEN]; // add 1k
          } catch (const std::bad_alloc &ex) {
            cerr << "oom: " << ex.what() << endl;
            exit(-1);
          }

          if (c_index_buffer_[word_id].pbuffer) {
            memcpy(pstr, c_index_buffer_[word_id].pbuffer,
                c_index_buffer_[word_id].pos); //TODO: memmove
            delete[] c_index_buffer_[word_id].pbuffer;
            c_index_buffer_[word_id].pbuffer = NULL;
          }

          c_index_buffer_[word_id].pbuffer = pstr;
          c_index_buffer_[word_id].size += LBUFFERLEN; //new capacity
          pstr = NULL;
        }
        //now, spaces are sufficiently
        memcpy(c_index_buffer_[word_id].pbuffer + c_index_buffer_[word_id].pos,
            tmpbuf, tmplen); // write to buffer
        c_index_buffer_[word_id].pos += tmplen;
      }
    }
  } else if (field == T_TITLE) {
    //TODO
    //Segment::get_instance().segment(text.c_str(), res);
    seg_->segment(text.c_str(), res);
    for (it = res.begin(); it != res.end(); ++it) {
      if (!(it->type == 'C' || it->type == 'c' || it->type == 'x'
          || it->type == 'S'))
        continue;

      int word_id = words_ht_->get_id(it->word.c_str());
      if (word_id == -1) {
        char* tmp = new char[it->word.length() + 1];
        strcpy(tmp, it->word.c_str());

        word_id = words_ht_->insert(tmp, word_seq_);
        pre_word_seq_ = word_seq_;
        if (word_seq_ > MAX_WORDSNUM_SIZE) {
          cerr << "word_seq_ > MAX_WORDSNUM_SIZE" << endl;
          exit(-1);
        }
      }
      int inter = doc_seq - pre_doc_seq_t_[word_id];
      pre_doc_seq_t_[word_id] = doc_seq;

      char tmpbuf[16]; //TODO: size
      int tmplen = 0;

      index_compr_->encode(inter, tmpbuf, tmplen, useless); // default: BYTECODEEX

      if (tmplen + t_index_buffer_[word_id].pos > GBUFFERLEN) {
        write_buffer(t_index_buffer_, word_id, tmpbuf, tmplen, tidx_path_);
      } else {
        if (tmplen + t_index_buffer_[word_id].pos
            > t_index_buffer_[word_id].size) { //not enough space
          char* pstr = NULL;
          try {
            pstr = new char[t_index_buffer_[word_id].size + LBUFFERLEN]; // add 1k
          } catch (const std::bad_alloc &ex) {
            cerr << "oom: " << ex.what() << endl;
            exit(-1);
          }

          if (t_index_buffer_[word_id].pbuffer) {
            memcpy(pstr, t_index_buffer_[word_id].pbuffer,
                t_index_buffer_[word_id].pos); //TODO: memmove
            delete[] t_index_buffer_[word_id].pbuffer;
            t_index_buffer_[word_id].pbuffer = NULL;
          }

          t_index_buffer_[word_id].pbuffer = pstr;
          t_index_buffer_[word_id].size += LBUFFERLEN; //new capacity
          pstr = NULL;
        }
        //now, spaces are sufficiently
        memcpy(t_index_buffer_[word_id].pbuffer + t_index_buffer_[word_id].pos,
            tmpbuf, tmplen); // write to buffer
        t_index_buffer_[word_id].pos += tmplen;
      }
    }
  } else {
    cerr << " FieldType ERROR!" << endl;
    return;
  }
}

void Indexer::write_buffer(SBuffer* index_buffer,
                           int id,
                           const char* pstr,
                           int len,
                           const string& idx_path) {
  int remain_size = GBUFFERLEN - index_buffer[id].pos;
  memcpy(index_buffer[id].pbuffer + index_buffer[id].pos, pstr, remain_size); //fullfill

  ostringstream oss;
  oss << idx_path; // endwith '/'
  oss << id % PARTITION_SIZE << "/" << id;
  ofstream outf(oss.str().c_str(), std::ofstream::app);
  if (!outf) {
    cerr << "open file " << oss.str() << " error" << endl;
    exit(-1);
  }
  outf.write(index_buffer[id].pbuffer, GBUFFERLEN); //write 4k
  index_buffer[id].pos = 0; //RESET

  memcpy(index_buffer[id].pbuffer + index_buffer[id].pos, pstr + remain_size,
      len - remain_size); //the remainent data
  index_buffer[id].pos += len - remain_size;

  outf.close();
}

void Indexer::flush(SBuffer* index_buffer, const string& idx_path) {
  for (int id = 0; id < word_seq_; ++id) {
    if (index_buffer[id].pos == 0)
      continue;

    ostringstream oss;
    oss << idx_path;
    oss << id % PARTITION_SIZE << "/" << id;

    ofstream outf(oss.str().c_str(), std::ofstream::app);
    if (!outf) {
      cerr << "open file error" << endl;
      exit(-1);
    }

    outf.write(index_buffer[id].pbuffer, index_buffer[id].pos); //write buf to disk
    outf.close();
  }

}

void Indexer::merge(const string& idx_file, const string& idx_path) {
  FILE* f_index = fopen(idx_file.c_str(), "wb");
  if (!f_index) {
    cerr << "open file error" << endl;
    exit(-1);
  }
  int onedata = 0;

  const int k_buf_size = 4096 * 1024;

  for (int i = 0; i < word_seq_ + 1; ++i) {
    fwrite(&onedata, 1, sizeof(int), f_index);
  }

  int offset = ftell(f_index);
  char* buf = NULL;
  try {
    buf = new char[k_buf_size];
    index_offset_ = new int[word_seq_]; //lazy new
  } catch (const std::bad_alloc &ex) {
    cerr << "oom: " << ex.what() << endl;
    exit(-1);
  }

  for (int i = 0; i < word_seq_; ++i) {
    FILE *fin;
    ostringstream oss;
    oss << idx_path;
    oss << i % PARTITION_SIZE << "/" << i;

    if (!(fin = fopen(oss.str().c_str(), "rb"))) {
      index_offset_[i] = offset;
      continue;
    }

    int buflen = k_buf_size;
    fseek(fin, 0, SEEK_SET);

    do {
      buflen = fread(buf, sizeof(char), k_buf_size, fin);
      fwrite(buf, sizeof(char), buflen, f_index);
    } while (buflen == k_buf_size);

    fclose(fin);
    offset = ftell(f_index); // record end position
    index_offset_[i] = offset;
  } //end for

  delete[] buf;
  buf = NULL;

  onedata = (word_seq_ + 1) * sizeof(int);
  fseek(f_index, 0, SEEK_SET);
  fwrite(&onedata, sizeof(int), 1, f_index);
  fseek(f_index, sizeof(int), SEEK_SET);
  fwrite(index_offset_, sizeof(int), word_seq_, f_index);
  fclose(f_index);

  delete[] index_offset_;
  index_offset_ = NULL;

}

void Indexer::complete() const {
  string word_info_file = Config::get_instance().get("word_info_file");
  words_ht_->save(word_info_file);
  
  string doc_info_file = Config::get_instance().get("doc_info_file");
  save_doc_info(doc_info_file);

}

void Indexer::save_doc_info(const string& file) const {
  ofstream fout(file.c_str(),ofstream::out);
  if (!fout) {
    cerr<<"can not open"<<file<<endl;
    return;
  }
  
  fout << doc_seq_ <<endl;
  fout << total_doc_len_ << endl;
  for (int i=0;i<doc_seq_;++i) {
    fout << doc_info_[i].doc_len <<" "
	 << doc_info_[i].url <<"\n"
	 << doc_info_[i].title << endl;
  }
  
  fout.close();
}

void Indexer::stat() const {
  cout << "words: " << word_seq_ << endl;
  cout << "docs: " << doc_seq_ << endl;
  cout << "used time: " << used_time_ << endl;
}
